<?php
/**
 * Created by PhpStorm.
 * User: 44766
 * Date: 2018-11-15
 * Time: 17:49
 */

namespace App\Services\CrawlerService;


use DB;
use Yangqi\Htmldom\Htmldom;

class HtmlService
{
    protected $name;
    protected $prefix;
    protected $list_url;
    protected $list_page;
    protected $list_regular;
    protected $list_range   = 1;
    protected $detail_rules = [];
    protected $detail_data  = [];
    protected $detail_urls  = [];


    public function getAllMd5()
    {
        return [];
    }

    public function setPage($page)
    {
        $this->list_range = $page;
        return $this;
    }

    public function getListUrls()
    {
        if (is_numeric($this->list_range)) {
            return [str_replace('{page}', $this->list_range, $this->list_url)];
        }

        list($min, $max) = $this->list_range;
        $pages = [];
        for ($i = $min; $i <= $max; $i++) {
            $pages[] = str_replace('{page}', $i, $this->list_url);
        };
        return $pages;
    }

    public function getListPage()
    {
        $this->detail_urls = collect();
        foreach ($this->getListUrls() as $url) {
            $content = $this->getContent($url);
            $html    = new Htmldom();
            $html    = $html->str_get_html($content);
            $list    = $html->find($this->list_regular);

            $this->detail_urls = $this->detail_urls->merge(array_map(function ($item) {
                return $this->prefix . $item->href;
            }, $list));
        }
        return $this;
    }

    public function getList()
    {
        return $this->detail_urls;
    }

    public function saveList()
    {
        DB::table('crawlers')->insert($this->detail_urls);
    }

    public function getDetails($urls)
    {
        $details = [];
        foreach ($urls as $url) {
            $this->getDetail($url);
            $details[] = $this->detail_data;
        }
        $this->detail_data = $details;
        return $this;
    }

    public function getDetail($url)
    {
        $content = $this->getContent($url);
        $this->analyzeContent($content);
        return $this;
    }

    public function getDetailData()
    {
        return $this->detail_data;
    }

    public function analyzeContent($content)
    {

        $html              = new Htmldom();
        $html              = $html->str_get_html($content);
        if($html){
            $this->detail_data = [];
            foreach ($this->detail_rules as $key => $rule) {
                $list = $html->find($rule['node']);

                $render_method = "renderField" . ucfirst(camel_case($key));
                if (!method_exists($this, $render_method)) {
                    $render_method = "renderField";
                }
                if (isset($rule['multi']) && $rule['multi'] === true) {
                    $this->detail_data[$key] = [];
                    foreach ($list as $node) {
                        $this->detail_data[$key][] = $this->$render_method($node, $rule);
                    }
                } else {
                    if (isset($list[0])) {
                        $this->detail_data[$key] = $this->$render_method($list[0], $rule);
                    } else {
                        $this->detail_data[$key] = null;
                    }
                }
            }
            $this->afterCrawl($this->detail_data);
        }
        return $this;
    }

    public function save()
    {
    }

    private function renderField($node, $rule)
    {
        if (isset($rule['attr']) && $rule['attr']) {
            $text = $node->{$rule['attr']};
        } else {
            $text = $node->innertext;
        }
        $this->autoFilter($text);
        return $text;
    }

    public function getContent($url, $javascript_loop = 0, $timeout = 5)
    {
        $url = str_replace("&amp;", "&", urldecode(trim($url)));

        $ch = curl_init();
        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1");
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
        curl_setopt($ch, CURLOPT_ENCODING, "");
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($ch, CURLOPT_AUTOREFERER, true);
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);    # required for https urls
        curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, $timeout);
        curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
        curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
        $content  = curl_exec($ch);
        $response = curl_getinfo($ch);
        curl_close($ch);

        if ($response['http_code'] == 301 || $response['http_code'] == 302) {
            ini_set("user_agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1");

            if ($headers = get_headers($response['url'])) {
                foreach ($headers as $value) {
                    if (substr(strtolower($value), 0, 9) == "location:")
                        return $this->getContent(trim(substr($value, 9, strlen($value))));
                }
            }
        }

        if ((preg_match("/>[[:space:]]+window\.location\.replace\('(.*)'\)/i", $content, $value) || preg_match("/>[[:space:]]+window\.location\=\"(.*)\"/i", $content, $value)) &&
            $javascript_loop < 5
        ) {
            return $this->getContent($value[1], $javascript_loop + 1);
        } else {
//            return [$content, $response];
            return $content;
        }
    }

    public function autoFilter(&$text)
    {
        $text = trim(html_entity_decode(strip_tags($text))," \t\n\r\0\x0B\xC2\xA0");
    }

    protected function afterCrawl(&$data)
    {
    }
}